{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# XGBoost Parameter Tuning for Rental Listing Inquiries Dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 第五步：调整正则化参数：reg_alpha 和reg_lambda"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "首先 import 必要的模块"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from xgboost import XGBClassifier\n",
    "import xgboost as xgb\n",
    "\n",
    "import pandas as pd \n",
    "import numpy as np\n",
    "\n",
    "import math\n",
    "\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "\n",
    "from sklearn.metrics import log_loss\n",
    "\n",
    "from matplotlib import pyplot\n",
    "import seaborn as sns\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 读取数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "dpath = './data/'\n",
    "train = pd.read_csv(dpath +\"RentListingInquries_FE_train.csv\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Variable Identification"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "选择该数据集是因为的数据特征单一，我们可以在特征工程方面少做些工作，集中精力放在参数调优上"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Target 分布，看看各类样本分布是否均衡"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZYAAAEKCAYAAAAxXHOuAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAGodJREFUeJzt3X+wHWWd5/H3hwCCAibClY1JMKyb\nUSNqhLshyuygiBCYXQn+mIJ1JGVRG8cNiqVlCZYalcEf5aA1mRFmYpElTDnGiD+IbtgYEXF15UfA\nCITI5E4AiUmROEEIOqKBz/7Rz4WTcHJPJ+lzT87N51XVdfp8++k+365bybe6n6eflm0iIiKaclCv\nE4iIiLElhSUiIhqVwhIREY1KYYmIiEalsERERKNSWCIiolEpLBER0agUloiIaFQKS0RENOrgXifQ\nC8ccc4ynTp3a6zQiIvrGHXfc8WvbA3XaHpCFZerUqaxevbrXaURE9A1JD9Ztm1thERHRqK4WFkmH\nSbpN0s8lrZX0yRK/RtL9ktaUZUaJS9JCSUOS7pJ0Ysux5kpaX5a5LfGTJN1d9lkoSd08p4iIGFm3\nb4U9AZxm+3FJhwA/lnRD2fYh29ft0v4sYFpZTgauAk6W9AJgATAIGLhD0nLbj5Q284BbgBXAbOAG\nIiKiJ7p6xeLK4+XrIWUZaZ7+c4Bry363AOMlTQTOBFbZ3laKySpgdtl2lO2fupr//1pgTtdOKCIi\nOup6H4ukcZLWAFuoisOtZdPl5XbXFyU9p8QmAQ+17L6xxEaKb2wTb5fHPEmrJa3eunXrPp9XRES0\n1/XCYvtJ2zOAycBMSScAlwIvA/4z8ALgw6V5u/4R70W8XR6LbA/aHhwYqDViLiIi9sKojQqz/Rvg\nh8Bs25vL7a4ngP8FzCzNNgJTWnabDGzqEJ/cJh4RET3S7VFhA5LGl/XDgdOBX5S+EcoIrjnAPWWX\n5cAFZXTYLOBR25uBlcAZkiZImgCcAaws27ZLmlWOdQFwfTfPKSIiRtbtUWETgSWSxlEVsWW2vyvp\nB5IGqG5lrQH+qrRfAZwNDAG/A94FYHubpMuA20u7T9neVtbfA1wDHE41GiwjwiIiekjVYKoDy+Dg\noPPkfcT+7ZS/O6XXKYx5P3nvT2q3lXSH7cE6bfPkfURENCqFJSIiGpXCEhERjUphiYiIRqWwRERE\no1JYIiKiUSksERHRqBSWiIhoVApLREQ0KoUlIiIalcISERGNSmGJiIhGpbBERESjUlgiIqJRKSwR\nEdGoFJaIiGhUCktERDQqhSUiIhqVwhIREY1KYYmIiEalsERERKO6WlgkHSbpNkk/l7RW0idL/HhJ\nt0paL+lrkg4t8eeU70Nl+9SWY11a4vdJOrMlPrvEhiRd0s3ziYiIzrp9xfIEcJrtVwMzgNmSZgGf\nA75oexrwCHBhaX8h8Ijt/wR8sbRD0nTgPOAVwGzgSknjJI0DvgScBUwHzi9tIyKiR7paWFx5vHw9\npCwGTgOuK/ElwJyyfk75Ttn+Rkkq8aW2n7B9PzAEzCzLkO0Ntv8ALC1tIyKiR7rex1KuLNYAW4BV\nwL8Cv7G9ozTZCEwq65OAhwDK9keBo1vju+yzu3hERPRI1wuL7SdtzwAmU11hvLxds/Kp3Wzb0/iz\nSJonabWk1Vu3bu2ceERE7JVRGxVm+zfAD4FZwHhJB5dNk4FNZX0jMAWgbH8+sK01vss+u4u3+/1F\ntgdtDw4MDDRxShER0Ua3R4UNSBpf1g8HTgfWATcBbyvN5gLXl/Xl5Ttl+w9su8TPK6PGjgemAbcB\ntwPTyiizQ6k6+Jd385wiImJkB3dusk8mAkvK6K2DgGW2vyvpXmCppL8GfgZcXdpfDfyTpCGqK5Xz\nAGyvlbQMuBfYAcy3/SSApIuAlcA4YLHttV0+p4iIGEFXC4vtu4DXtIlvoOpv2TX+e+DtuznW5cDl\nbeIrgBX7nGxERDQiT95HRESjUlgiIqJRKSwREdGoFJaIiGhUCktERDQqhSUiIhqVwhIREY0a8TkW\nSR8YabvtLzSbTkRE9LtOD0geOSpZRETEmDFiYbH9ydFKJCIixoZafSyS/kTSjZLuKd9fJemj3U0t\nIiL6Ud3O+y8DlwJ/hKfnADuvW0lFRET/qltYnmv7tl1iO9q2jIiIA1rdwvJrSS+hvJ1R0tuAzV3L\nKiIi+lbdafPnA4uAl0n6FXA/8I6uZRUREX2rbmF50Pbpkp4HHGR7ezeTioiI/lX3Vtj9khZRva/+\n8S7mExERfa5uYXkp8H2qW2L3S/p7SX/avbQiIqJf1Sostv/d9jLbb6F61fBRwM1dzSwiIvpS7Uko\nJZ0q6UrgTuAw4C+6llVERPStWp33ku4H1gDLgA/Z/m1Xs4qIiL5V94rl1bbPtf3VPSkqkqZIuknS\nOklrJV1c4p+Q9CtJa8pydss+l0oaknSfpDNb4rNLbEjSJS3x4yXdKmm9pK9JOrRufhER0by6heU/\n7OVcYTuAD9p+OdWIsvmSppdtX7Q9oywrynGnU00V8wpgNnClpHGSxgFfAs4CpgPntxznc+VY04BH\ngAtrnlNERHRBV+cKs73Z9p1lfTuwDpg0wi7nAEttP2H7fmAImFmWIdsbbP8BWAqcI0nAacB1Zf8l\nwJya5xQREV0wanOFSZpKNaLs1hK6SNJdkhZLmlBik4CHWnbbWGK7ix8N/Mb2jl3iERHRI6MyV5ik\nI4BvAO+3/RhwFfASYEY5zhXDTdvs7r2It8thnqTVklZv3bq1buoREbGH9mWusL+ss6OkQ6iKylds\nfxPA9sMt278MfLd83QhMadl9MrCprLeL/xoYL+ngctXS2n4ntheVc2BwcLBt8YmIiH1X9wHJDbZP\nBwaAl9n+U9sPdNqv9IFcDayz/YWW+MSWZucC95T15cB5kp4j6XhgGnAbcDswrYwAO5Sqf2e5bQM3\nAW8r+88Frq9zThER0R0jXrFI+sBu4gC0FovdOAV4J3C3pDUl9hGqUV0zqG5bPQC8uxxvraRlwL1U\nfTjzbT9ZfvMiYCUwDlhse2053oeBpZL+GvgZVSGLiIge6XQr7Mh9ObjtH9O+H2TFCPtcDlzeJr6i\n3X62N1CNGouIiP3AiIXF9ifrHETSpbY/00xKERHRz2rPFdbB2xs6TkRE9LmmCku7210REXEAaqqw\nZPhuREQAuWKJiIiG1Soskk7pEPt6YxlFRERfq3vF8ncjxWx/upl0IiKi33V6QPK1wOuAgV0eljyK\n6kHFiIiInXR6QPJQ4IjSrvVhycd4ZhqViIiIp3V6QPJm4GZJ19h+EEDSQcARZZbiiIiIndTtY/mM\npKMkPY9qHq/7JH2oi3lFRESfqltYppcrlDlU83UdRzW5ZERExE7qFpZDyntV5gDX2/4jeSgyIiLa\nqFtY/pFqevvnAT+S9GKqDvyIiIid1HqDpO2FwMKW0IOS3tCdlCIiop/VffL+WElXS7qhfJ9O9bbG\niIiIndS9FXYN1dsbX1S+/wvw/m4kFBER/a1uYTnG9jLgKQDbO4Anu5ZVRET0rbqF5beSjqaMBJM0\nC3i0a1lFRETfqtV5D3wAWA68RNJPgAEypUtERLTRsbCUKVwOA04FXkr17pX7yrMsERERO+l4K8z2\nU8AVtnfYXmv7nrpFRdIUSTdJWidpraSLS/wFklZJWl8+J5S4JC2UNCTpLkknthxrbmm/XtLclvhJ\nku4u+yyUlJeORUT0UN0+lu9Jeute/Ke9A/ig7ZcDs4D5ZajyJcCNtqcBN5bvAGcB08oyD7gKqkIE\nLABOBmYCC4aLUWkzr2W/2XuYY0RENKhuYfkA1Vsi/yDpMUnbJXV88t72Ztt3lvXtwDpgEnAOsKQ0\nW0I1VQwlfq0rtwDjJU0EzgRW2d5m+xFgFTC7bDvK9k9tG7i25VgREdEDdZ+8P7Jzq5FJmgq8BrgV\nONb25nLszZJeWJpNAh5q2W1jiY0U39gmHhERPVL3yXtJ+ktJHyvfp0iaWfdHJB0BfAN4f4f3uLS7\n1ea9iLfLYZ6k1ZJWb926tVPKERGxl+reCrsSeC3w38v3x4Ev1dmxzIr8DeArtr9Zwg+X21iUzy0l\nvhGY0rL7ZGBTh/jkNvFnsb3I9qDtwYGBgTqpR0TEXqhbWE62PR/4PUDp5zi0006ls/9qYJ3tL7Rs\nWs4zc43NBa5viV9QrpBmAY+WW2YrgTMkTSid9mcAK8u27ZJmld+6oOVYERHRA3UfkPyjpHE88+T9\nAGV6lw5OoXoh2N2S1pTYR4DPAsskXQj8Enh72bYCOBsYAn4HvAvA9jZJlwG3l3afsr2trL+Hai6z\nw4EbyhIRET1St7AsBL4FvFDS5VRP3X+s0062f0z7fhCAN7Zpb2D+bo61GFjcJr4aOKFTLhERMTrq\njgr7iqQ7qIqBgDm213U1s4iI6Eu1Coukf7L9TuAXbWIRERFPq9t5/4rWL6W/5aTm04mIiH43YmGR\ndKmk7cCrWp643041PDijryIi4llGLCy2P1Oeuv+87aNsH1mWo21fOko5RkREH6nbeX+ppEnAi1v3\nsf2jbiUWERH9qW7n/WeB84B7eeaVxAZSWCIiYid1n2M5F3ip7Se6mUxERPS/uqPCNgCHdDORiIgY\nG+pesfwOWCPpRuDpqxbb7+tKVhER0bfqFpblZYmIiBhR3VFhSzq3ioiI6FBYJC2z/ReS7qbNC7Rs\nv6prmUVERF/qdMVycfn8r91OJCIixoYRC0vLe+kfHJ10IiKi39UdbhwREVFLCktERDSq0+zGN5bP\nz41OOhER0e86dd5PlHQq8GZJS9nlNcO27+xaZhER0Zc6FZaPA5cAk4Ev7LLNwGndSCoiIvpXp1Fh\n1wHXSfqY7ctGKaeIiOhjdZ+8v0zSm4E/K6Ef2v5u99KKiIh+VWtUmKTPUD0seW9ZLi6xTvstlrRF\n0j0tsU9I+pWkNWU5u2XbpZKGJN0n6cyW+OwSG5J0SUv8eEm3Slov6WuSDq132hER0S11hxv/OfAm\n24ttLwZml1gn15S2u/qi7RllWQEgaTrVy8ReUfa5UtI4SeOALwFnAdOB80tbgM+VY00DHgEurHk+\nERHRJXvyHMv4lvXn19mhvLp4W83jnwMstf2E7fuBIWBmWYZsb7D9B2ApcI4kUQ0euK7svwSYU/O3\nIiKiS+oWls8AP5N0jaQlwB3Ap/fhdy+SdFe5VTahxCYBD7W02Vhiu4sfDfzG9o5d4m1JmidptaTV\nW7du3YfUIyJiJLUKi+2vArOAb5bltbaX7uVvXgW8BJgBbAauKHG1aeu9iLdle5HtQduDAwMDe5Zx\nRETUVvdFX8MTUu7zy75sPzy8LunLwPDoso3AlJamk4FNZb1d/NfAeEkHl6uW1vYREdEjoz5XmKSJ\nLV/PBYZHjC0HzpP0HEnHA9OA24DbgWllBNihVB38y20buAl4W9l/LnD9aJxDRETsXu0rlr0h6avA\n64FjJG0EFgCvlzSD6rbVA8C7AWyvlbSMajjzDmC+7SfLcS4CVgLjgMW215af+DCwVNJfAz8Dru7m\n+URERGcdC4ukg4C7bJ+wpwe3fX6b8G7/87d9OXB5m/gKYEWb+AaqUWMREbGf6HgrzPZTwM8lHTcK\n+URERJ+reytsIrBW0m3Ab4eDtt/clawiIqJv1S0sn+xqFhERMWbUnYTyZkkvBqbZ/r6k51J1pEdE\nROyk7iSU/4Nq6pR/LKFJwLe7lVRERPSvus+xzAdOAR4DsL0eeGG3koqIiP5Vt7A8USaABEDSwYww\nfUpERBy46haWmyV9BDhc0puArwPf6V5aERHRr+oWlkuArcDdVE/KrwA+2q2kIiKif9UdFfZUmS7/\nVqpbYPeVuboiIiJ2UquwSPpz4B+Af6Warv54Se+2fUM3k4uIiP5T9wHJK4A32B4CkPQS4H8DKSwR\nEbGTun0sW4aLSrEB2NKFfCIios+NeMUi6S1lda2kFcAyqj6Wt1O9JyUiImInnW6F/beW9YeBU8v6\nVmDCs5tHRMSBbsTCYvtdo5VIRESMDXVHhR0PvBeY2rpPps2PiIhd1R0V9m2qNz9+B3iqe+lENOeX\nn3plr1M4IBz38bt7nULsZ+oWlt/bXtjVTCIiYkyoW1j+VtIC4HvAE8NB23d2JauIiOhbdQvLK4F3\nAqfxzK0wl+8RERFPq/uA5LnAf7R9qu03lKVjUZG0WNIWSfe0xF4gaZWk9eVzQolL0kJJQ5LuknRi\nyz5zS/v1kua2xE+SdHfZZ6Ek1T/1iIjohrqF5efA+L04/jXA7F1ilwA32p4G3Fi+A5wFTCvLPOAq\nqAoRsAA4GZgJLBguRqXNvJb9dv2tiIgYZXULy7HALyStlLR8eOm0k+0fAdt2CZ8DLCnrS4A5LfFr\nXbkFGC9pInAmsMr2NtuPAKuA2WXbUbZ/WmZavrblWBER0SN1+1gWNPibx9reDGB7s6ThVxxPAh5q\nabexxEaKb2wTb0vSPKqrG4477rh9PIWIiNiduu9jubnbiVBNx/+sn96LeFu2FwGLAAYHB/MumYiI\nLql1K0zSdkmPleX3kp6U9Nhe/ubD5TYW5XN4luSNwJSWdpOBTR3ik9vEIyKih2oVFttH2j6qLIcB\nbwX+fi9/czkwPLJrLnB9S/yCMjpsFvBouWW2EjhD0oTSaX8GsLJs2y5pVhkNdkHLsSIiokfq9rHs\nxPa3JV3SqZ2krwKvB46RtJGqr+azwDJJFwK/pJqCH2AFcDYwBPwOeFf5rW2SLuOZafo/ZXt4QMB7\nqEaeHU710rG8eCwiosfqTkL5lpavBwGDjNCfMcz2+bvZ9MY2bQ3M381xFgOL28RXAyd0yiMiIkZP\n3SuW1vey7AAeoBoeHBERsZO6o8LyXpaIiKil06uJPz7CZtu+rOF8IiKiz3W6Yvltm9jzgAuBo4EU\nloiI2EmnVxNfMbwu6UjgYqrRWkuBK3a3X0REHLg69rGUSSA/ALyDam6vE8ucXREREc/SqY/l88Bb\nqKZCeaXtx0clq4iI6Fudnrz/IPAi4KPAppZpXbbvw5QuERExhnXqY6k7rX5ERARQ/30sERERtaSw\nREREo1JYIiKiUSksERHRqBSWiIhoVApLREQ0KoUlIiIalcISERGNSmGJiIhGpbBERESjUlgiIqJR\nKSwREdGonhUWSQ9IulvSGkmrS+wFklZJWl8+J5S4JC2UNCTpLkknthxnbmm/XtLcXp1PRERUen3F\n8gbbM2wPlu+XADfangbcWL4DnAVMK8s84Cp4+iVkC4CTgZnAguFiFBERvdHrwrKrc6jeUkn5nNMS\nv9aVW4DxkiYCZwKrbG8rb7VcBcwe7aQjIuIZvSwsBr4n6Q5J80rsWNubAcrnC0t8EvBQy74bS2x3\n8YiI6JGO77zvolNsb5L0QmCVpF+M0FZtYh4h/uwDVMVrHsBxxx23p7lGRERNPbtisb2pfG4BvkXV\nR/JwucVF+dxSmm8EprTsPhnYNEK83e8tsj1oe3BgYKDJU4mIiBY9KSySnifpyOF14AzgHmA5MDyy\nay5wfVlfDlxQRofNAh4tt8pWAmdImlA67c8osYiI6JFe3Qo7FviWpOEc/tn2/5F0O7BM0oXAL4G3\nl/YrgLOBIeB3wLsAbG+TdBlwe2n3KdvbRu80IiJiVz0pLLY3AK9uE/834I1t4gbm7+ZYi4HFTecY\nERF7Z38bbhwREX0uhSUiIhrVy+HGfeGkD13b6xTGvDs+f0GvU4iIBuWKJSIiGpXCEhERjUphiYiI\nRqWwREREo1JYIiKiUSksERHRqBSWiIhoVApLREQ0KoUlIiIalcISERGNSmGJiIhGpbBERESjUlgi\nIqJRKSwREdGoFJaIiGhUCktERDQqhSUiIhqVwhIREY0aE4VF0mxJ90kaknRJr/OJiDiQ9X1hkTQO\n+BJwFjAdOF/S9N5mFRFx4Or7wgLMBIZsb7D9B2ApcE6Pc4qIOGCNhcIyCXio5fvGEouIiB44uNcJ\nNEBtYn5WI2keMK98fVzSfV3NqneOAX7d6yT2hP5mbq9T2J/03d+PBe3+CR6w+urvp/ft0d/uxXUb\njoXCshGY0vJ9MrBp10a2FwGLRiupXpG02vZgr/OIvZO/X3/L368yFm6F3Q5Mk3S8pEOB84DlPc4p\nIuKA1fdXLLZ3SLoIWAmMAxbbXtvjtCIiDlh9X1gAbK8AVvQ6j/3EmL/dN8bl79ff8vcDZD+rnzsi\nImKvjYU+loiI2I+ksIwhmdqmf0laLGmLpHt6nUvsGUlTJN0kaZ2ktZIu7nVOvZZbYWNEmdrmX4A3\nUQ3Bvh043/a9PU0sapH0Z8DjwLW2T+h1PlGfpInARNt3SjoSuAOYcyD/28sVy9iRqW36mO0fAdt6\nnUfsOdubbd9Z1rcD6zjAZ/9IYRk7MrVNRI9Jmgq8Bri1t5n0VgrL2FFrapuI6A5JRwDfAN5v+7Fe\n59NLKSxjR62pbSKieZIOoSoqX7H9zV7n02spLGNHpraJ6AFJAq4G1tn+Qq/z2R+ksIwRtncAw1Pb\nrAOWZWqb/iHpq8BPgZdK2ijpwl7nFLWdArwTOE3SmrKc3eukeinDjSMiolG5YomIiEalsERERKNS\nWCIiolEpLBER0agUloiIaFQKS0TDJI2X9D9H4XdeL+l13f6diD2VwhLRvPFA7cKiyt78W3w9kMIS\n+508xxLRMEnDM0vfB9wEvAqYABwCfNT29WWywhvK9tcCc4DTgQ9TTcWzHnjC9kWSBoB/AI4rP/F+\n4FfALcCTwFbgvbb/72icX0QnKSwRDStF47u2T5B0MPBc249JOoaqGEwDXgxsAF5n+xZJLwL+H3Ai\nsB34AfDzUlj+GbjS9o8lHQestP1ySZ8AHrf9N6N9jhEjObjXCUSMcQI+XV7k9RTVqwyOLdsetH1L\nWZ8J3Gx7G4CkrwN/UradDkyvpqQC4KjyQqmI/VIKS0R3vQMYAE6y/UdJDwCHlW2/bWnX7rUHww4C\nXmv731uDLYUmYr+SzvuI5m0Hhq8ong9sKUXlDVS3wNq5DThV0oRy++ytLdu+RzXBKACSZrT5nYj9\nRgpLRMNs/xvwE0n3ADOAQUmrqa5efrGbfX4FfJrqzYPfB+4FHi2b31eOcZeke4G/KvHvAOeW2XT/\nS9dOKGIPpfM+Yj8h6Qjbj5crlm8Bi21/q9d5ReypXLFE7D8+IWkNcA9wP/DtHucTsVdyxRIREY3K\nFUtERDQqhSUiIhqVwhIREY1KYYmIiEalsERERKNSWCIiolH/H0mmMnB4UroEAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x80cdba8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "sns.countplot(train.interest_level);\n",
    "pyplot.xlabel('target');\n",
    "pyplot.ylabel('Number of interest_level');"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "每类样本分布不是很均匀，所以交叉验证时也考虑各类样本按比例抽取"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# drop ids and get labels\n",
    "y_train = train['interest_level']\n",
    "\n",
    "train = train.drop([\"interest_level\"], axis=1)\n",
    "X_train = np.array(train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "各类样本不均衡，交叉验证是采用StratifiedKFold，在每折采样时各类样本按比例采样"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# prepare cross validation\n",
    "kfold = StratifiedKFold(n_splits=5, shuffle=True, random_state=3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "第二轮参数调整得到的n_estimators最优值（218），其余参数继续默认值"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "用交叉验证评价模型性能时，用scoring参数定义评价指标。评价指标是越高越好，因此用一些损失函数当评价指标时，需要再加负号，如neg_log_loss，neg_mean_squared_error 详见sklearn文档：http://scikit-learn.org/stable/modules/model_evaluation.html#log-loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'reg_alpha': [1.5, 2], 'reg_lambda': [0.5, 1, 2]}"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#reg_alpha = [1e-3, 1e-2, 0.05, 0.1]    #default = 0\n",
    "#reg_lambda = [1e-3, 1e-2, 0.05, 0.1]   #default = 1\n",
    "\n",
    "reg_alpha = [ 1.5, 2]    #default = 0, 测试0.1,1，1.5，2\n",
    "reg_lambda = [0.5, 1, 2]      #default = 1，测试0.1， 0.5， 1，2\n",
    "\n",
    "param_test5_1 = dict(reg_alpha=reg_alpha, reg_lambda=reg_lambda)\n",
    "param_test5_1\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Anaconda\\lib\\site-packages\\sklearn\\model_selection\\_search.py:761: DeprecationWarning: The grid_scores_ attribute was deprecated in version 0.18 in favor of the more elaborate cv_results_ attribute. The grid_scores_ attribute will not be available from 0.20\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "([mean: -0.58132, std: 0.00367, params: {'reg_alpha': 1.5, 'reg_lambda': 0.5},\n",
       "  mean: -0.58127, std: 0.00342, params: {'reg_alpha': 1.5, 'reg_lambda': 1},\n",
       "  mean: -0.58169, std: 0.00337, params: {'reg_alpha': 1.5, 'reg_lambda': 2},\n",
       "  mean: -0.58148, std: 0.00331, params: {'reg_alpha': 2, 'reg_lambda': 0.5},\n",
       "  mean: -0.58189, std: 0.00298, params: {'reg_alpha': 2, 'reg_lambda': 1},\n",
       "  mean: -0.58188, std: 0.00336, params: {'reg_alpha': 2, 'reg_lambda': 2}],\n",
       " {'reg_alpha': 1.5, 'reg_lambda': 1},\n",
       " -0.58126624322974696)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xgb5_1 = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=218,  #第二轮参数调整得到的n_estimators最优值\n",
    "        max_depth=6,\n",
    "        min_child_weight=4,\n",
    "        gamma=0,\n",
    "        subsample=0.8,\n",
    "        colsample_bytree=0.6,\n",
    "        colsample_bylevel = 0.7,\n",
    "        objective= 'multi:softprob',\n",
    "        seed=3)\n",
    "\n",
    "\n",
    "gsearch5_1 = GridSearchCV(xgb5_1, param_grid = param_test5_1, scoring='neg_log_loss',n_jobs=-1, cv=kfold)\n",
    "gsearch5_1.fit(X_train , y_train)\n",
    "\n",
    "gsearch5_1.grid_scores_, gsearch5_1.best_params_,     gsearch5_1.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Anaconda\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('mean_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Anaconda\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split0_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Anaconda\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split1_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Anaconda\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split2_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Anaconda\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split3_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Anaconda\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split4_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Anaconda\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('std_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'mean_fit_time': array([  533.04819994,   509.80900002,  1481.78579998,   858.38439999,\n",
       "          557.60860009,   458.14139996]),\n",
       " 'mean_score_time': array([ 1.69300003,  1.61100001,  1.69720006,  1.55939999,  1.41019998,\n",
       "         1.23559999]),\n",
       " 'mean_test_score': array([-0.58131847, -0.58126624, -0.58169456, -0.58148055, -0.58188694,\n",
       "        -0.58187664]),\n",
       " 'mean_train_score': array([-0.47534476, -0.47706296, -0.47949674, -0.47800435, -0.47935712,\n",
       "        -0.48158479]),\n",
       " 'param_reg_alpha': masked_array(data = [1.5 1.5 1.5 2 2 2],\n",
       "              mask = [False False False False False False],\n",
       "        fill_value = ?),\n",
       " 'param_reg_lambda': masked_array(data = [0.5 1 2 0.5 1 2],\n",
       "              mask = [False False False False False False],\n",
       "        fill_value = ?),\n",
       " 'params': [{'reg_alpha': 1.5, 'reg_lambda': 0.5},\n",
       "  {'reg_alpha': 1.5, 'reg_lambda': 1},\n",
       "  {'reg_alpha': 1.5, 'reg_lambda': 2},\n",
       "  {'reg_alpha': 2, 'reg_lambda': 0.5},\n",
       "  {'reg_alpha': 2, 'reg_lambda': 1},\n",
       "  {'reg_alpha': 2, 'reg_lambda': 2}],\n",
       " 'rank_test_score': array([2, 1, 4, 3, 6, 5]),\n",
       " 'split0_test_score': array([-0.57488594, -0.57549528, -0.57564066, -0.57556862, -0.57694234,\n",
       "        -0.57593524]),\n",
       " 'split0_train_score': array([-0.47583618, -0.47734856, -0.48031593, -0.47880776, -0.48024768,\n",
       "        -0.48170995]),\n",
       " 'split1_test_score': array([-0.58010033, -0.5799075 , -0.58069923, -0.58060944, -0.57997322,\n",
       "        -0.58059765]),\n",
       " 'split1_train_score': array([-0.47504488, -0.47727423, -0.4797194 , -0.47780111, -0.47879221,\n",
       "        -0.48115587]),\n",
       " 'split2_test_score': array([-0.58241584, -0.58173277, -0.5828641 , -0.5824936 , -0.583576  ,\n",
       "        -0.58320746]),\n",
       " 'split2_train_score': array([-0.47519519, -0.47702226, -0.47932862, -0.47728285, -0.47985041,\n",
       "        -0.48179524]),\n",
       " 'split3_test_score': array([-0.58367993, -0.5839057 , -0.5841122 , -0.58351374, -0.58433711,\n",
       "        -0.58441313]),\n",
       " 'split3_train_score': array([-0.47557097, -0.47631691, -0.47942283, -0.47822978, -0.47983696,\n",
       "        -0.48169407]),\n",
       " 'split4_test_score': array([-0.5855116 , -0.58529119, -0.58515766, -0.58521848, -0.58460686,\n",
       "        -0.58523075]),\n",
       " 'split4_train_score': array([-0.47507656, -0.47735283, -0.47869694, -0.47790026, -0.47805833,\n",
       "        -0.48156884]),\n",
       " 'std_fit_time': array([   1.82339525,    4.65398066,  797.58784735,  637.64673524,\n",
       "          23.06304416,   78.24003553]),\n",
       " 'std_score_time': array([ 0.1832835 ,  0.1937431 ,  0.21663004,  0.14616241,  0.0732103 ,\n",
       "         0.19240436]),\n",
       " 'std_test_score': array([ 0.00366669,  0.0034215 ,  0.00337183,  0.00331183,  0.00297879,\n",
       "         0.0033583 ]),\n",
       " 'std_train_score': array([ 0.00030883,  0.00039204,  0.00052803,  0.00050379,  0.00080903,\n",
       "         0.00022633])}"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gsearch5_1.cv_results_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best: -0.581266 using {'reg_alpha': 1.5, 'reg_lambda': 1}\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZgAAAELCAYAAADkyZC4AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzt3Xd4FVX6wPHvm0YIoUmHAKGE3gmI\noCC4roAKq2BBRcGCrt3F3pYf6oro2hZdFRV7IypNig1EFJSwkNAhIEKoIdTQ0t7fH3PBa0i5KTdz\nk7yf58nDnZkzM++E3Pvec87MOaKqGGOMMSUtyO0AjDHGlE+WYIwxxviFJRhjjDF+YQnGGGOMX1iC\nMcYY4xeWYIwxxviFJRhjjDF+YQnGGGOMX1iCMcYY4xchbgfgptq1a2t0dLTbYRhjTJmybNmyvapa\np6ByFTrBREdHEx8f73YYxhhTpojI776U82sTmYgMFJH1IpIkIg/msn2UiKSIyArPz41e2yaKyGoR\nWSsiL4uIeNY/JSLbRCQtx7Gaish3IpIoIgtEJMqf12aMMSZ/fkswIhIMvAIMAtoBI0SkXS5FP1XV\nLp6fNz379gb6AJ2ADkAPoJ+n/EygZy7HeQ54T1U7AeOBp0vyeowxxhSOP2swPYEkVd2squnAJ8BQ\nH/dVIBwIAyoBocBuAFVdoqo7c9mnHfCd5/X8QpzLGGOMH/gzwTQCtnktJ3vW5TTM06wVJyKNAVR1\nMU6S2On5maeqaws4XwIwzPP6EqCqiNQqzgUYY4wpOn8mGMllXc7JZ2YC0Z5mrW+BdwFEpCXQFojC\nSUoDRKRvAee7F+gnIstxmtO2A5mnBSUyRkTiRSQ+JSWlMNdjjDGmEPyZYJKBxl7LUcAO7wKqmqqq\nJzyLk4HunteXAEtUNU1V04A5QK/8TqaqO1T1UlXtCjziWXcwl3JvqGqsqsbWqVPgXXbGGGOKyJ8J\nZikQIyLNRCQMuBKY4V1ARBp4LQ4BTjaDbcWpjYSISChOjSTfJjIRqS0iJ6/nIeDtErgGY4wxReS3\n52BUNVNEbgfmAcHA26q6WkTGA/GqOgO4U0SG4DRl7QNGeXaPAwYAK3Ga1eaq6kxwbl8GrgIiRCQZ\neFNVxwHnAk+LiAILgdv8dW0mQG36Hvb9BsGhEBTi+Qn2ep3Xj6fMqf1y7hMMQTmOKbm1ABtjvIlq\nzm6RiiM2NlbtQctyYtXnEHd96Z1PvJJQcB4JKyjEk5hyS1ghXkktr+15JbxinDPPJJpHIs253RKr\nAURkmarGFlSuQj/Jb8qJPWth+h3Q+EwYPgVQyM6ErEzn31M/WV6vM3JZ51nOyshjn5PL+Ww/7Zze\nZTL+eJ15/PTtWRl5nM8rXs1293ddokmtgISWbxItwdron86Zy3VIkCXWIrIEY8q24wfhk6shrApc\n9i5Ua1DwPmVZdjZoQUktZyLNmbRyJrWSSKS5nSOXxJ1xzLdEmnN7QCTWfJJWvs2yxUmkviRuX5Jv\nLueMqA3h1fz6a7MEY8qu7Gz48u+wfwtcN7P8JxeAoCAgyPlwqUhOJtZ8a5f5JL1ck5oPibTAGq0P\n58xMh+yjPiZ/78Sf4d/f6YXPQ48b/HoKSzCm7PrpBVj/FVzwNET3cTsa408VObHmm9B8SaR5JLVG\n3Qs+fzFZgjFl06bv4fsnof2l0OvvbkdjjH8EBUFQGM6oWWWPTThmyp4DWyHuBqjdGob8xzpgjQlQ\nlmBM2ZJxHD4d6VTxr/gAKkW6HZExJg/WRGbKltn3ws4VcOVHULul29EYY/JhNRhTdix7F5a/D+eM\nhTYXuh2NMaYAlmBM2bB9mVN7ad4f+j/idjTGGB9YgjGB78he+PRaiKwHw95yHhozxgQ864MxgS07\nyxlj7EgK3DAPqtgccsaUFZZgTGD7/gn47QcYMgkadnU7GmNMIVgTmQlca2fCoheg+yjoNtLtaIwx\nhWQJxgSmvRudccYadoNBE92OxhhTBJZgTOA5kQafXgMhYXDF+xBSye2IjDFFYH0wJrCowvTbYO8G\nGPklVI9yOyJjTBFZDcYElsWTYM00OO9xaH6u29EYY4rBEowJHL/9CN/8E9peDH3udjsaY0wxWYIx\ngeHgdogbDWc0h6Gv2gjJxpQD1gdj3Jd5AqZeB+lH4bpZfp/G1RhTOizBGPfNexiSl8Jl70DdNm5H\nY4wpIdZEZty14mNY+ib0vgPaX+J2NMaYEuTXBCMiA0VkvYgkiciDuWwfJSIpIrLC83Oj17aJIrJa\nRNaKyMsiTqO8iDwlIttEJC3HsZqIyHwRWS4iiSIy2J/XZkrAzgSYdTdEnwPnjXM7GmNMCfNbghGR\nYOAVYBDQDhghIu1yKfqpqnbx/Lzp2bc30AfoBHQAegD9POVnAj1zOc6jwGeq2hW4Eni1JK/HlLCj\n+5yZKSufAcOnQLC11hpT3vjzXd0TSFLVzQAi8gkwFFjjw74KhANhgAChwG4AVV3iOV5u+5zsHa4O\n7Che+MZvsrPhizFwaAeMngORddyOyBjjB/5sImsEbPNaTvasy2mYp0krTkQaA6jqYmA+sNPzM09V\n1xZwvnHANSKSDMwG7ihm/MZffpgASd/AoAnQuIfb0Rhj/MSfCSa3Bxk0x/JMIFpVOwHfAu8CiEhL\noC0QhZOUBohI3wLONwJ4R1WjgMHA+yJy2vWJyBgRiReR+JSUlEJdkCkB6+fCD89A56sg9ga3ozHG\n+JE/E0wy0NhrOYoczVaqmqqqJzyLk4HunteXAEtUNU1V04A5QK8CzncD8JnnuItxmthq5yykqm+o\naqyqxtapY00zpWrfZvhyDNTvCBc9bw9TGlPO+TPBLAViRKSZiIThdLzP8C4gIg28FocAJ5vBtgL9\nRCREREJxOvgLaiLbCpznOW5bnARjVZRAkX7U6dRH4IoPILSy2xEZY/zMbwlGVTOB24F5OMnhM1Vd\nLSLjRWSIp9idnluRE4A7gVGe9XHAJmAlkAAkqOpMOHX7cjIQISLJIjLOs89Y4CbPsT4GRqlqziY5\n4wZV53bk3ath2FtQM9rtiIwxpUAq8mdwbGysxsfHux1G+ffLGzDnPuj/CPS73+1ojDHFJCLLVDW2\noHL2JL/xr61LYN5D0GognHOv29EYY0qRJRjjP4d3w2fXQfXGcMnrEGR/bsZUJPb4tPGPrAyYOgqO\nH4RrPofKNdyOyBhTyizBGP/45nHY+jNcOhnqd3A7GmOMC6zNwpS8lXGw5FXoeTN0utztaIwxLrEE\nY0rW7jUw4w5o3Av++qTb0RhjXGQJxpSc4wfh02ugUlW4/F0ICXM7ImOMi6wPxpSM7Gz48hY48Dtc\nNxOq1nc7ImOMyyzBmJKx6HlYPxsGToCmvd2OxhgTAKyJzBRf0nfw/ZPQYTiceYvb0RhjAoQlGFM8\n+3+Hz2+Aum1hyMs2QrIx5hRLMKboMo7DZyMhO8sZITmsitsRGWMCiPXBmKJRhdljYWcCXPkx1Grh\ndkTGmABjNRhTNMvegeUfOANYthnsdjTGmABkCcYUXvIymHM/tDgP+j/sdjTGmABlCcYUzpG98Nm1\nEFkfhr0JQcFuR2SMCVDWB2N8l5UJcaPhSArc8DVEnOF2RMaYAGYJxvju+yfgt4Uw9BVo2MXtaIwx\nAc6ayIxv1syAn16E7qOh6zVuR2OMKQMswZiCpWyAaX+HRt1h0DNuR2OMKSMswZj8nTjsjJAcEg6X\nvwchldyOyBhTRlgfjMmbKky/DVI3wshpUD3K7YiMMWWI1WBM3n7+D6yZDuf9E5r3czsaY0wZ49cE\nIyIDRWS9iCSJyIO5bB8lIikissLzc6PXtokislpE1orIyyLOKIoi8pSIbBORtBzHesHrOBtE5IA/\nr63c+20hfPtPaDsE+tzldjTGmDLIb01kIhIMvAKcDyQDS0VkhqquyVH0U1W9Pce+vYE+QCfPqkVA\nP2ABMBOYBGz03kdV7/Ha/w6ga4ldTEVzcDtMHQ21WsLfXrURko0xReLPGkxPIElVN6tqOvAJMNTH\nfRUIB8KASkAosBtAVZeo6s4C9h8BfFykqCu6zBPOk/qZx50RkitVdTsiY0wZ5c8E0wjY5rWc7FmX\n0zARSRSROBFpDKCqi4H5wE7PzzxVXevLSUWkKdAM+L44wVdYcx+C7fHOw5R1WrsdjTGmDPNngsmt\nXUVzLM8EolW1E/At8C6AiLQE2gJROElpgIj09fG8VwJxqpqVa1AiY0QkXkTiU1JSfDxkBbHiI4h/\nC3rfCe3/5nY0xhg/WbX9IBlZ2X4/jz8TTDLQ2Gs5CtjhXUBVU1X1hGdxMtDd8/oSYImqpqlqGjAH\n6OXjea8kn+YxVX1DVWNVNbZOnTo+HrIC2JkAs+6B6HOcu8aMMeXOicwsnpm7jiGTFjHlp9/8fj5/\nJpilQIyINBORMJwP/hneBUSkgdfiEOBkM9hWoJ+IhIhIKE4Hf4FNZCLSGqgJLC6B+CuOo/uchykj\nasHwKRBsj0cZU94kJh/g4v8s4r8LNjG8exRX9mzi93P67ZNEVTNF5HZgHhAMvK2qq0VkPBCvqjOA\nO0VkCJAJ7ANGeXaPAwYAK3Ga1eaq6kxwbl8GrgIiRCQZeFNVx3n2GwF8oqo5m+JMXrKz4PMb4fAu\nGD0HIq1WZ0x5kp6ZzX++38irCzZROzKMKaN60L9N3VI5t1Tkz+LY2FiNj493Owx3ff8ULJwIF70A\nsde7HY0xpgSt3nGQsZ8lsG7XYS7t1oh/XtSe6hGhxT6uiCxT1diCyllbSEW2fo6TXLpc7YySbIwp\nFzKysnl1/ib+8/1GalYJY/K1sZzfrl6px2EJpqJK3QRf3AwNOsOF/7aHKY0pJ9btOsTYzxJYveMQ\nQ7s0ZNzF7alZJcyVWCzBVETpR+DTkRAUBJe/D6GV3Y7IGFNMmVnZvL5wMy9+u4Fq4aG8dk03BnZo\nUPCOfmQJpqJRhZl3wZ41cE0c1GzqdkTGmGLauPswY6cmkJh8kAs7NmD80PbUinR/ag1LMBXNr2/A\nyqnQ/1Fo+Re3ozHGFENWtjL5x808//UGqlQKZtJVXbmoU0O3wzrFEkxF8vtimPcwtBoE54x1Oxpj\nTDFsSknj3qkJLN96gAva1+PJv3WkTlX3ay3eLMFUFId3wdTroEYTuOQ1p//FGFPmZGUrU376jWfn\nrSc8NJiXruzCkM4NkQC8UccSTEWQlQFTRznTH4/8EirXcDsiY0wRbNl7hPviEli6ZT/ntanL05d2\npG61cLfDypMlmIrg68dg62IY9hbUa+92NMaYQsrOVt5bvIUJc9cRGhzEvy/rzKXdGgVkrcWbJZjy\nbmUc/PJfOPMW6Djc7WiMMYW0NfUo98Ul8Mtv+zi3dR0mXNqJ+tUDt9bizRJMebZ7Ncy4A5qcBX99\n0u1ojDGFkJ2tfPjrVp6evZYgESYO68RlsVEBX2vxZgmmvDp2wBkhuVJVuOwdCC7++EPGmNKRvP8o\nD3yeyE9JqZwTU5sJwzrRqEbZeyC6UAlGRIKASFU95Kd4TEnIzoYvb4EDW+G6WVC1vtsRGWN8oKp8\nsnQbT321lmxVnrqkA1f1bFKmai3eCkwwIvIRcAuQBSwDqovI86r6rL+DM0W06N+wYQ4MfAaanuV2\nNMYYH+w4cIwHv1jJwg0pnNW8FhOHd6LxGRFuh1UsvtRg2qnqIRG5GpgNPICTaCzBBKKkb50h+Dte\nBmfe7HY0xpgCqCpxy5IZP3MNmdnK+KHtuebMpgQFlc1aizdfEkyoZ1bJvwGTVDVDRCruJDKBbP/v\nzuRhddvBxS/ZCMnGBLjdh47z0Bcr+X7dHnpGn8Gzl3Wiaa0qbodVYnxJMK8DW4AEYKGINAWsDybQ\nZBxzOvWzs+GK9yGs/PyRGlPeqCrTVmznn9NXk56VzeMXtWNU7+hyUWvxVmCCUdWXgZe9Vv0uIv39\nF5IpNFX4aizsSoQRn0CtFm5HZIzJw57Dx3nky1V8s2Y33ZrU4LnLOtO8TqTbYfmFL538dwFTgMPA\nm0BX4EHga/+GZny2bAqs+BD63getB7kdjTEmF6rKzMSdPD59FUfTs3h4cBtuOLs5weWs1uLNlyay\n61X1JRG5AKgDjMZJOJZgAkFyPMy+H1qcB+c+5HY0xphcpKad4NFpq5izahedG9fg35d1omXdqm6H\n5Xe+JJiT6XUwMEVVE6Ss3pRd3qSlwGfXQrUGMOxNCAp2OyJjTA5zVu7k0WmrOHw8k/sHtmbMOc0J\nCa4Yo5n7kmCWicjXQDPgIRGpCmT7NyxToKxMiBsNR1Phhq8h4gy3IzLGeNl/JJ3HZ6xmZsIOOjaq\nznOXdaZ1/fJfa/HmS4K5AegCbFbVoyJSC6eZzLjpu/+DLT/C0FehQWe3ozHGePl69S4e/nIVB4+l\nM/b8VtxybgtCK0itxVuBV6yq2UAU8KiIPAf0VtVEXw4uIgNFZL2IJInIg7lsHyUiKSKywvNzo9e2\niSKyWkTWisjLJ5vlROQpEdkmImm5HO9yEVnj2e8jX2Isk9ZMh59fhtjroevVbkdjjPE4cDSdez5d\nwZj3l1GnaiWm33Y2d5wXUyGTC/h2F9kEoAfwoWfVnSLSW1Xz7VEWkWDgFeB8IBlYKiIzVHVNjqKf\nqurtOfbtDfQBOnlWLQL6AQuAmcAkYGOOfWKAh4A+qrpfROoWdG1lUsp6mHYrNIqFgRPcjsYY4/H9\nut08+PlKUo+kc+d5MdzevyVhIRUzsZzkSxPZYKCLpyaDiLwLLMf5MM9PTyBJVTd79vsEGArkTDC5\nUSAcCMO5ySAU2A2gqks8x8u5z03AK6q631Nujw/nKVtOHHYepgwJh8vfg5DAmn/bmIro4LEMnpy1\nhqnLkmldrypvj+pBh0bV3Q4rIPg6mnINYJ/nta+/uUbANq/lZODMXMoNE5G+wAbgHlXdpqqLRWQ+\nsBMnwUxS1bUFnK8VgIj8BAQD41R1ro+xBj5Vp+aSmgTXTofqjdyOyJgK74cNKTz4eSK7Dx3ntv4t\nuPO8GCqF2N2cJ/mSYJ4Glns+8AXoS8G1F/jj9mZvOccwmwl8rKonROQW4F1ggIi0BNri9P0AfCMi\nfVV1YT7nCwFigHM9+/0oIh1U9cCfghIZA4wBaNKkiQ+XESB+fhnWzoDzn4Bmfd2OxpgK7fDxDP41\ney0f/7qNlnUj+fLWPnRuXMPtsAKOL0PFfCwiC3D6YQRnNGVfGhaTgcZey1HAjhzHTvVanAw843l9\nCbBEVdMARGQO0AvIL8Eke/bJAH4TkfU4CWdpjnO+AbwBEBsbWzYG7dz8A3w7DtoNhd53uB2NMRXa\noo17eeDzRHYePMbN/Zpzz19aER5qtZbc+NQDpao7VXWGqk5X1V3AEh92WwrEiEgzEQkDrgRmeBcQ\nkQZei0OAk81gW4F+IhLiGcm5n9e2vEwD+nuOWxunyWyzD3EGtoPJEHc91IqBoa/YCMnGuOTIiUwe\nnbaSa976hUohQUy9pTcPDWprySUfRZ0yucBPOVXNFJHbgXk4fSJvq+pqERkPxKvqDJw70oYAmTh9\nPKM8u8cBA4CVOM1qc1V1Jji3LwNXAREikgy8qarjPOf5q4iswZkc7b4cNaSyJ/OE86R+5nG44gNn\n+mNjTKlbvCmV++IS2H7gGDee3Yx7L2hticUHolr4ViIR2aqqZagDI3exsbEaHx/vdhh5m3m3M5Dl\n5e85zWPGmFJ1ND2TiXPX887PW2haK4LnLutMj2gbNUNElqlqbEHl8qzBiMh/OL1THpzai/Vm+dvy\nD5zk0ucuSy7GuGDpln3cOzWB31OPMqp3NPcPbE1EWFEbfSqm/H5b+X21D+Cv/eXAjhUw6x/O3WID\nHnc7GmMqlOMZWTw7bz1v//QbjWpU5uObenFWi1puh1Um5ZlgVPXd0gzEeBzdB5+OhCq1YdjbEGzf\nmIwpLct+3899UxPYvPcI1/RqwkOD2lKlkr0Hi8p+c4EkOws+vxHSdsHouRBZx+2IjKkQjmdk8cK3\nG5i8cDMNqlfmgxvO5OyY2m6HVeZZggkkC56GTd/BRS9CVHe3ozGmQkjYdoCxUxNI2pPGiJ6NeXhw\nW6qGh7odVrlgCSZQrJsNC5+FLtdA91FuR2NMuXciM4uXv9vIaz9spk5kJd69vif9WlmrQUnyZTTl\nl3NZfRDnWZbpJR9SBZS6Cb682ZnX5cLn7GFKY/xs1faDjP0sgfW7DzO8exSPXdSO6pWt1lLSfKnB\nhANtgKme5WHAauAGEemvqnf7K7gKIf2IM0JyUDBc/j6EVnY7ImPKrfTMbCbNT+KV+UnUqhLGW9fF\ncl7bem6HVW75kmBaAgNUNRNARP4LfI0zz8tKP8ZW/qnCjDthz1q45nOo2dTtiIwpt9bsOMS9UxNY\ns/MQl3RtxD8vbkeNiDC3wyrXfEkwjYAqOM1ieF43VNUsETnht8gqgl9eg1VxMOBRaHme29EYUy5l\nZGXz2oJNvPz9RqpXDuX1kd25oH19t8OqEHxJMBOBFZ4RlU8O1/8vEakCfOvH2Mq333+Grx+F1oPh\n7LFuR2NMubR+12HunZrAyu0HubhzQ/5vSHvOqGK1ltLiy3D9b4nIbJwZKgV4WFVPDrt/nz+DK7cO\n74Kpo6BGE/jbfyGoYk+rakxJy8zK5o0fN/PiNxuJDA/h1au7Mbhjg4J3NCXK19uUewDneF5nkWNe\nF1MImenw2XXO9Mcjp0FlG9bNmJKUtOcwY6cmkrDtAIM61OeJv3WgdqRNL+4GX25TnoCTYD70rLpT\nRHqrqi+zWpqcvnkMti2BYW9BvXZuR2NMuZGVrby1aDPPfb2BiLBgXh7RlYs7NUDstn/X+FKDGQx0\nUdVsABF5F1iOb9MmG2+JU52O/V63QsfhbkdjTLmxOSWN++ISWfb7fs5vV4+nLulA3arhbodV4fna\nRFYDZ0IwgOp+iqV827UKZtwBTXrD+ePdjsaYciE7W3nn5y1MnLeOsOAgXriiM3/r0shqLQHClwTz\nNLBcRObzx11kVnspjGMHnIcpw6vDZe9AsD0xbExx/Z56hPumJvLrln30b12HCcM6Ua+a1VoCiS93\nkX3suUW5B06CeQCw2558lZ3tDANzcBuM+gqq2lPDxhRHdrbywS+/8/TsdYQECc8O78Tw7lFWawlA\nPjWRqepOYMbJZRHZCpT5KZNLxY/PwYa5MGgiNOnldjTGlGnb9h3l/rhEFm9OpW+rOky4tCMNa9jw\nSoGqqKMp21cFX2z8Fub/CzpeDj3HuB2NMWWWqvLRr1v511drEREmXNqRK3o0tlpLgCtqgtESjaI8\n2r8FPr8B6rWHi1+yEZKNKaLtB47x4OeJ/LhxL31a1uKZYZ2IqhnhdljGB3kmGBH5D7knEsG5q8zk\nJeOY06mvCpe/B2H2ZjCmsFSVz+K38cSstWSr8uTfOnD1mU2s1lKG5FeDiS/itopNFWb9A3athBGf\nQq0WbkdkTJmz8+AxHvx8JT9sSOHMZmfw7PDONKllX9TKmjwTjKq+m3OdiNRX1V2+HlxEBgIvAcHA\nm6o6Icf2UcCzwHbPqkmq+qZn20TgQpw71r4B7lJVFZGngGuBmqoa6cuxSlX825DwEfR7AFoPLPXT\nG1OWqSpf/G8742auJiMrm3EXt+Pas6IJCrJaS1lU2D6Y2UA3XwqKSDDwCs68McnAUhGZoaprchT9\nVFVvz7Fvb6AP0MmzahHQD1gAzAQmARtzOe1pxypV25bCnAeg5V+cBGOM8dmeQ8d5+MuVfLt2D7FN\na/LcZZ2Jrl3F7bBMMRQ2wRTma0RPIElVNwOIyCfAUCBngsmN4sykGeY5ZyiwG0BVl3iOV4hQSkFa\nCnx2LVRrCJdOdmaoNMYUSFWZkbCDx6ev5nhGFo9e2JbRfZoRbLWWMq+wCWZyIco2ArZ5LScDZ+ZS\nbpiI9AU2APeo6jZVXewZOWAnToKZpKprfTjnaccqRLxFl5UJcaPh2D644WuIOKNUTmtMWZdy+ASP\nTlvJvNW76dqkBs9d1pkWdSIL3tGUCYV6Il9VXy1E8dy+fuS8K20mEK2qnXAmL3sXQERaAm2BKJxE\nNcCTOPKT67FOC0pkjIjEi0h8SkqKzxeTr+/GwZYf4aIXoEHnkjmmMeXcrMQd/PWFH5i/LoUHB7Uh\n7pbellzKGX8O+ZIMNPZajiLHPDKqmqqqJ6ddngx097y+BFiiqmmqmgbMAfJ9DD6fY+Us94aqxqpq\nbJ06dQp1Qbla/SX8/B+IvQG6XFX84xlTzu07ks5tH/6P2z9aTuMzIvjqzrO5pV8LaxIrh/yZYJYC\nMSLSTETCgCvxGm4GQES8p5gbApxsBtsK9BOREBEJxengz7eJLJ9j+U/Keph2G0T1gIETCi5vTAU3\nd9Uu/vrCD3y9Zhf3XdCaL/7em5h6Vd0Oy/hJUZ/kL5CqZorI7cA8nNuU31bV1SIyHohX1Rk4k5cN\nATJxpgMY5dk9DhgArMRpVpurqjPh1O3LVwERIpKMc/vzuHyO5R/HD8EnVzsPUV72LoTYPN/G5GX/\nkXTGzVzN9BU7aN+wGh/ceCZt6ldzOyzjZ6JacUd9iY2N1fj4IjwzqgqfjYR1s+Ha6dDsnIL3MaaC\n+mbNbh7+ciX7j6Rzx4AYbu3fgtBgG5C9LBORZaoaW1A5v9VgyrXFk2DtTPjrk5ZcjMnDwaMZ/N+s\n1Xzxv+20qV+Vd0b3oH1Dm6+wIrEEUxStBsGRvXCWe890GhPI5q/bw4NfJLI3LZ07BrTkjgExhIVY\nraWisQRTFLVbwvn/53YUxgScQ8czeHLWGj6LTyambiSTr42lU5SNjVtRWYIxxpSIHzem8EBcIrsO\nHefv57bg7r/EUCnERrSoyCzBGGOKJe1EJv+avZaPftlKizpV+PzvvenapKbbYZkAYAnGGFNkPyft\n5b64RHYcPMaYvs35x/mtCA+1WotxWIIxxhTakROZPDN3He8t/p1mtasQd8tZdG9qY/CZP7MEY4wp\nlF82p3JfXCLb9h/l+j7NuO+C1lQOs1qLOZ0lGGOMT46lZzFx3jre+XkLjWtG8MlNvTizeS23wzIB\nzBKMMaZA8Vv2cV9cIr/tPcJ1ZzXlgUFtiAizjw+TP/sLMcbk6XhGFv/+ej1vLvqNRjUq89FNZ9K7\nRW23wzJlhCUYY0yulm/dz9jN9lNJAAAbIUlEQVSpCWxOOcJVZzbh4cFtiaxkHxnGd/bXYoz5kxOZ\nWbz47UZe/2ET9auF8971PenbqgTmTjIVjiUYY8wpickHuHdqAht2p3FFbGMeuagt1cJD3Q7LlFGW\nYIwxpGdm85/vN/Lqgk3Ujgxjyuge9G9d1+2wTBlnCcaYCm7V9oPcOzWBdbsOM6xbFI9f1I7qEVZr\nMcVnCcaYCiojK5tX5icx6fskalYJY/K1sZzfrp7bYZlyxBKMMRXQul2HGPtZAqt3HGJol4aMu7g9\nNavYtN+mZFmCMaYCyczK5vWFm3nx2w1UCw/ltWu6MbBDA7fDMuWUJRhjKoiNuw8zdmoCickHubBT\nA8YPaU+tyEpuh2XKMUswxpRzWdnK5B838/zXG6hSKZhJV3Xlok4N3Q7LVACWYIwpxzalpHHv1ASW\nbz3ABe3r8eTfOlKnqtVaTOmwBGNMOZSVrUz56Teenbee8NBgXrqyC0M6N0RE3A7NVCBB/jy4iAwU\nkfUikiQiD+ayfZSIpIjICs/PjV7bJorIahFZKyIvi+edISJPicg2EUnL45zDRURFJNZ/V2ZM4Nqy\n9whXvrGYJ79ayzkxtfnmnr4M7dLIkospdX6rwYhIMPAKcD6QDCwVkRmquiZH0U9V9fYc+/YG+gCd\nPKsWAf2ABcBMYBKwMZdzVgXuBH4puSsxpmzIzlbeW7yFCXPXERocxL8v68yl3SyxGPf4s4msJ5Ck\nqpsBROQTYCiQM8HkRoFwIAwQIBTYDaCqSzzHy22/J4CJwL3FjN2YMmVr6lHui0vgl9/2cW7rOky4\ntBP1q4e7HZap4PzZRNYI2Oa1nOxZl9MwEUkUkTgRaQygqouB+cBOz888VV2b38lEpCvQWFVnFVBu\njIjEi0h8SkpKIS7HmMCTna28v+R3Br60kDU7DjFxWCemjOphycUEBH8mmNyqGJpjeSYQraqdgG+B\ndwFEpCXQFojCSUoDRKRvnicSCQJeAMYWFJSqvqGqsaoaW6eODUFuyq7k/UcZ+fYvPDZtFd2b1mTu\nPX25vEdjaxIzAcOfTWTJQGOv5Shgh3cBVU31WpwMPON5fQmwRFXTAERkDtALWJjHuaoCHYAFnjdX\nfWCGiAxR1fhiXocxAUVV+WTpNp76ai2qyr8u6ciInpZYTODxZw1mKRAjIs1EJAy4EpjhXUBEvMeo\nGAKcbAbbCvQTkRARCcXp4M+ziUxVD6pqbVWNVtVoYAlgycWUOzsOHOO6KUt56IuVdGxUnbl39+Wq\nM5tYcjEByW81GFXNFJHbgXlAMPC2qq4WkfFAvKrOAO4UkSFAJrAPGOXZPQ4YAKzEaVabq6ozwbl9\nGbgKiBCRZOBNVR3nr+swJhCoKlOXJfPEzDVkZitPDG3P1Wc2JSjIEosJXKKas1uk4oiNjdX4eKvk\nmMC2+9BxHvw8kfnrU+jZ7AyeHd6JprWquB2WqcBEZJmqFvisoT3Jb0yAUlW+XL6dcTNWk56VzeMX\ntWNU72irtZgywxKMMQFoz+HjPPLlKr5Zs5vuTWvy7PBONK8T6XZYxhSKJRhjAoiqMjNxJ49PX8XR\n9CweGdyW689uRrDVWkwZZAnGmACxN+0Ej01bxZxVu+jcuAb/vqwzLetarcWUXZZgjAkAs1fu5NFp\nq0g7nskDA9tw0znNCAn261i0xvidJRhjXLTvSDqPT1/FrMSddGxUnX9f3plW9aq6HZYxJcISjDEu\nmbd6F498uZKDxzIYe34rbjm3BaFWazHliCUYY0rZgaPp/N/MNXy5fDvtGlTjvevPpF3Dam6H5YqM\njAySk5M5fvy426GYXISHhxMVFUVoaGiR9rcEY0wp+m7tbh76YiX7jqRz13kx3Na/JWEhFbfWkpyc\nTNWqVYmOjrbhbgKMqpKamkpycjLNmjUr0jEswRhTCg4ey+CJWWuIW5ZMm/pVeXtUDzo0qu52WK47\nfvy4JZcAJSLUqlWL4kxrYgnGGD9bsH4PD36+kpS0E9zevyV3nNeSSiHBbocVMCy5BK7i/t9YgjHG\nTw4fz+Cpr9byydJttKwbyesju9O5cQ23wzKm1FTcxl9j/GjRxr0MfPFHPovfxs39mjPrjrMtuRgW\nLFjARRddVOwy+Vm3bh1nnXUWlSpV4rnnnsuz3KhRo2jWrBldunShS5curFixosjnzIvVYIwpQUdO\nZPL0nLV8sGQrzWtXYeotvenetKbbYRkfqSqqSlBQ2f3ufcYZZ/Dyyy8zbdq0Ass+++yzDB8+3G+x\nlN3fojEBZvGmVC54cSEf/rKVG89uxuy7zrHkUgZs2bKFtm3bcuutt9KtWzfef/99zjrrLLp168Zl\nl11GWloaALNnz6ZNmzacffbZ3HnnnfnWMn799Vd69+5N165d6d27N+vXrz+tzLhx4xg5ciQDBgwg\nJiaGyZMnn9qWlpbG8OHDadOmDVdffTUnp1UZP348PXr0oEOHDowZM4bcplupW7cuPXr0KPKtxSXJ\najDGFNPR9Ewmzl3POz9vIbpWBJ/dfBY9os9wO6wy5/9mrmbNjkMlesx2Davxz4vbF1hu/fr1TJky\nhfHjx3PppZfy7bffUqVKFZ555hmef/557r//fm6++WYWLlxIs2bNGDFiRL7Ha9OmDQsXLiQkJIRv\nv/2Whx9+mM8///y0comJiSxZsoQjR47QtWtXLrzwQgCWL1/O6tWradiwIX369OGnn37i7LPP5vbb\nb+fxxx8HYOTIkcyaNYuLL76Y1157DYBbbrmlUL+fRx55hPHjx3PeeecxYcIEKlWqVKj9C2IJxphi\nWLplH/dOTeD31KOM6h3N/QNbExFmb6uypmnTpvTq1YtZs2axZs0a+vTpA0B6ejpnnXUW69ato3nz\n5qeeBxkxYgRvvPFGnsc7ePAg1113HRs3bkREyMjIyLXc0KFDqVy5MpUrV6Z///78+uuv1KhRg549\nexIVFQVAly5d2LJlC2effTbz589n4sSJHD16lH379tG+fXsuvvjiQicWgKeffpr69euTnp7OmDFj\neOaZZ04lr5Ji7wRjiuB4RhbPzlvP2z/9RlTNynx8Uy/OalHL7bDKNF9qGv5SpYozQ6iqcv755/Px\nxx//afvy5csLdbzHHnuM/v378+WXX7JlyxbOPffcXMvlvA345LJ3TSI4OJjMzEyOHz/OrbfeSnx8\nPI0bN2bcuHHFGgGhQYMGp841evTofG8IKCrrgzGmkJb9vp/BL/3IW4t+45ozmzL3rr6WXMqJXr16\n8dNPP5GUlATA0aNH2bBhA23atGHz5s1s2bIFgE8//TTf4xw8eJBGjRoB8M477+RZbvr06Rw/fpzU\n1FQWLFhAjx498ix7MpnUrl2btLQ04uLiCnFlp9u5cyfgJNVp06bRoUOHYh0vN1aDKYL1uw6zbtch\nWtaNpEWdSMJD7aG5iuB4RhYvfLOByT9upkH1ynx445n0aVnb7bBMCapTpw7vvPMOI0aM4MSJEwA8\n+eSTtGrVildffZWBAwdSu3Ztevbsme9x7r//fq677jqef/55BgwYkGe5nj17cuGFF7J161Yee+wx\nGjZsyIYNG3ItW6NGDW666SY6duxIdHT0n5KRdx/Mrl27iI2N5dChQwQFBfHiiy+yZs0aqlWrxuDB\ng3nzzTdp2LAhV199NSkpKagqXbp0OXWMkiS53YVQUcTGxmp8fHyh93v5u408/43zRyACTc6IIKZu\nJC3rViWmbiQx9ZzEU6WS5e/yYsW2A9w7NYGkPWmM6NmEhwe3oWq4+3fplHVr166lbdu2bofhk7S0\nNCIjI1FVbrvtNmJiYrjnnnuKfLxx48YRGRnJvffeW4JRlrzc/o9EZJmqxha0r30CFsEt/VowsEN9\nNu5OY+Oew2zck0bS7jR+2JBCRtYfCbtRjcrE1It0kk7dqrSsF0nLupFUsw+mMuNEZhYvf7eR/y7Y\nRN2q4bx7fU/6tarjdljGBZMnT+bdd98lPT2drl27cvPNN7sdUsCzGkwRajB5ycjKZuu+o2zcnUaS\nJ/Fs3J3GppQ0TmRmnypXv1o4MZ5kE1O3qvO6TiQ1q4SVWCym+FZtP8jYzxJYv/swl3WP4tGL2lG9\nsn05KEllqQaTmylTpvDSSy/9aV2fPn145ZVXXIqo5AVsDUZEBgIvAcHAm6o6Icf2UcCzwHbPqkmq\n+qZn20TgQpwbEb4B7lJVFZGngGuBmqoa6XWsW4DbgCwgDRijqmv8eHmnCQ0OokUdp3kM6p9an5Wt\nJO8/6qnxOLWepD1pfLp0G0fTs06Vqx1Z6VQT26kmt3qR1KoSZgMClqL0zGwmzU/ilflJ1KoSxtuj\nYhnQpp7bYZkANHr0aEaPHu12GAHLbwlGRIKBV4DzgWRgqYjMyOVD/1NVvT3Hvr2BPkAnz6pFQD9g\nATATmARszHGcj1T1Nc/+Q4DngYEldkHFEBwkNK1Vhaa1qvCXdn98UGVnKzsOHjvVxHayue3L/23n\n8InMU+VqRISe1scTU7cq9apVssRTwtbsOMTYqQms3XmIS7s24p8Xt6d6hNVajCkKf9ZgegJJqroZ\nQEQ+AYYCvtQqFAgHwgABQoHdAKq6xHO8P++g6v0IcBXPMQJaUJAQVTOCqJoR9G9d99R6VWX3oRNO\nwvHUepL2HGbOqp18fPSPB7aqVgqhZY4+npi6kTSsXpmgIEs8hZGRlc1/F2zi5e82UiMijDdGduev\n7esXvKMxJk/+TDCNgG1ey8nAmbmUGyYifYENwD2quk1VF4vIfGAnToKZpKprCzqhiNwG/AMnMeV6\nb6CIjAHGADRp0qQQl1N6RIT61cOpXz2cc2L+6FBWVVKPpJ/Wx/P9uhQ+i08+VS4iLJiWdb36eDy1\nnqiaEQRb4jnN+l2HuXdqAiu3H+Tizg0ZP6S99YcZUwL8mWBy+yTLWauYCXysqic8fSjvAgNEpCXQ\nFojylPtGRPqq6sL8TqiqrwCviMhVwKPAdbmUeQN4A5xO/sJckNtEhNqRlagdWem0B/v2H0knKSXt\n1J1tSXvS+DkplS/+t/1UmUohTh9Rzj6epmdEEBJc8Z65zczK5o0fN/PiNxuJDA/h1au7MbhjA7fD\nMqbc8GeCSQYaey1HATu8C6hqqtfiZOAZz+tLgCWqmgYgInOAXkC+CcbLJ8B/ixBzmVWzShg9qpxx\n2iCLh45nkJSjjyd+y36mr/jjvyI0WGheO/JPzW0x9SKJrlWl3M4Xn7TnMGOnJpKw7QCDOtTnib91\noHZkyQ70Z0xOCxYs4LnnnmPWrFnFKpOfDz/8kGeecT5KIyMj+e9//0vnzp2LdKzi8meCWQrEiEgz\nnLvErgSu8i4gIg1UdadncQhwshlsK3CTiDyNUxPqB7yY38lEJEZVT3b8X8jpNwFUSNXCQ+nWpCbd\nmvx52PgjJzLZdKrG4zS5rdp+kNkrd3LyzvXgICG6VsQft1J7mt3K8ugFWdnKW4s289zXG4gIC+Y/\nI7pyUacGdrOEAcrHfDDNmjXjhx9+oGbNmsyZM4cxY8bwyy+/uBKL3xKMqmaKyO3APJzblN9W1dUi\nMh6IV9UZwJ2eO74ygX3AKM/ucTh9KCtxmtXmqupMOHX78lVAhIgk49z+PA64XUT+AmQA+8mlecz8\noUqlEDpF1aBT1J9nWTyekcWmlDSS9vzR3LZhz2G+WbubrGwn85TV0Qs2p6Rx79QE/rf1AOe3q8dT\nl3SgbtVwt8MyJ815EHatLNlj1u8IgybkW2TLli0MGjSI/v37s3jxYu6++25ee+01Tpw4QYsWLZgy\nZQqRkZHMnj2bf/zjH9SuXZtu3bqxefPmPGsZv/76K3fffTfHjh2jcuXKTJkyhdatW/+pzLhx49i0\naRPbt29n27Zt3H///dx0003AH/PBrFq1iu7du/PBBx8gIowfP56ZM2dy7Ngxevfuzeuvv37al6Pe\nvXufet2rVy+Sk5Nxi18/DVR1NjA7x7rHvV4/BDyUy35ZQK6Pyarq/cD9uay/q7jxGggPDaZ9w+q0\nb1j9T+tPZGaxZe/RU3e2JaWUndELsrOVKT9vYeLcdVQKCeLFK7owtEtDq7WYU8rrfDBvvfUWgwYN\nKolfUZEE7tdNE1AqhQTTun5VWtev+qf1eY1esHhTaoGjF8TUjaRGhH/v1vo99Qj3TU3k1y37GNCm\nLk9f2pF61azWEpAKqGn4U3mcD2b+/Pm89dZbLFq0qLi/niKzBGOKJVBHL8jOVj745Xeenr2OkCDh\n2eGdGN49ymotJlflbT6YxMREbrzxRubMmUOtWu5NJWEJxvhFcUcvqBkR+qeHR0/WeupWLXj0gm37\njnJ/XCKLN6fSt1UdnhnWkQbVK/vtWk350atXL2677TaSkpJo2bIlR48eJTk5+U/zwURHR5fofDAP\nPfQQR44cYcGCBUyYMCHP4fpzmw9m+PDhp5XbunUrl156Ke+//z6tWrXy8cr9wxKMKVWFHb3gq8Sd\nHDzmNXpBeMifEs7JO9saVq+MCHz061b+9dVaRIQJl3bkih6NrdZifFYe5oMZP348qamp3HrrrQCE\nhIRQkoP6FoaNpuzSL974RlXZm5Z+qonN+0HSvWnpp8pFhAVTKzKMbfuO0adlLZ4Z1omomhEuRm58\nUZZGU7b5YP4QEKMpG1NcIkKdqpWoU7USvVv8efbInKMX/J56lFv6teCqnk2s1mJKnM0HU3hWg7Ea\njDGuKUs1mNzYfDD5sxqMMcYUkc0Hk7+yOx6CMaZcqMitKIGuuP83lmCMMa4JDw8nNTXVkkwAUlVS\nU1MJDy/6g8nWRGaMcU1UVBTJycmkpKS4HYrJRXh4+KkRBYrCEowxxjWhoaGnhl8x5Y81kRljjPEL\nSzDGGGP8whKMMcYYv6jQD1qKSArwu9txeKkN7HU7iHwEenwQ+DEGenwQ+DEGenxQ/mNsqqp1CipU\noRNMoBGReF+ejnVLoMcHgR9joMcHgR9joMcHFuNJ1kRmjDHGLyzBGGOM8QtLMIEl7zlYA0OgxweB\nH2OgxweBH2OgxwcWI2B9MMYYY/zEajDGGGP8whJMKRORgSKyXkSSROTBPMpcLiJrRGS1iHwUaDGK\nSBMRmS8iy0UkUUQGl3J8b4vIHhFZlcd2EZGXPfEniki3AIvvak9ciSLys4h0Ls34fInRq1wPEckS\nkdMnf/czX2IUkXNFZIXnvfJDIMUnItVFZKaIJHjiK9Vx/UWksed9utZz/rtyKePf94qq2k8p/QDB\nwCagORAGJADtcpSJAZYDNT3LdQMwxjeAv3tetwO2lHKMfYFuwKo8tg8G5gAC9AJ+CbD4env9/w4q\n7fh8idHrb+F7YDYwPNBiBGoAa4AmnuXSfq8UFN/DwDOe13WAfUBYKcbXAOjmeV0V2JDLe9mv7xWr\nwZSunkCSqm5W1XTgE2BojjI3Aa+o6n4AVd0TgDEqUM3zujqwoxTjQ1UX4rxZ8zIUeE8dS4AaItKg\ndKIrOD5V/fnk/y+wBCj6cLVF5MPvEOAO4HOgtP8GAZ9ivAr4QlW3esqXapw+xKdAVXHm7470lM0s\njdgAVHWnqv7P8/owsBZolKOYX98rlmBKVyNgm9dyMqf/h7cCWonITyKyREQGllp0Dl9iHAdcIyLJ\nON9u7yid0HzmyzUEihtwvkEGFBFpBFwCvOZ2LPloBdQUkQUiskxErnU7oBwmAW1xvoCtBO5S1Ww3\nAhGRaKAr8EuOTX59r9hw/aVLclmX8za+EJxmsnNxvtn+KCIdVPWAn2M7yZcYRwDvqOq/ReQs4H1P\njK68eXLhyzW4TkT64ySYs92OJRcvAg+oapbzBTwghQDdgfOAysBiEVmiqhvcDeuUC4AVwACgBfCN\niPyoqodKMwgRicSpid6dy7n9+l6xBFO6koHGXstRnN68lAwsUdUM4DcRWY+TcJaWTog+xXgDMBBA\nVReLSDjOuEauNKXkwpdrcJWIdALeBAapaqrb8eQiFvjEk1xqA4NFJFNVp7kb1p8kA3tV9QhwREQW\nAp1x+hoCwWhggjqdHUki8hvQBvi1tAIQkVCc5PKhqn6RSxG/vlesiax0LQViRKSZiIQBVwIzcpSZ\nBvQHEJHaOM0AmwMsxq043xoRkbZAOBBIUxLOAK713CHTCzioqjvdDuokEWkCfAGMDKBv23+iqs1U\nNVpVo4E44NYASy4A04FzRCRERCKAM3H6GQKF9/ukHtCaUnwve/p+3gLWqurzeRTz63vFajClSFUz\nReR2YB7OHTpvq+pqERkPxKvqDM+2v4rIGiALuK80v+H6GONYYLKI3INTnR7l+ZZWKkTkY5wmxNqe\nfqB/AqGe+F/D6RcaDCQBR3G+SZYaH+J7HKgFvOqpIWRqKQ+M6EOMrisoRlVdKyJzgUQgG3hTVfO9\n7bo04wOeAN4RkZU4TVEPqGppjrDcBxgJrBSRFZ51DwNNvGL063vFnuQ3xhjjF9ZEZowxxi8swRhj\njPELSzDGGGP8whKMMcYYv7AEY4wxxi8swRhjjPELSzDGlAGeYelnFbeMMaXJEowxJcDzJLS9n4zx\nYm8IY4pIRKI9kzm9CvwPGCkii0XkfyIy1TPIICIyWETWicgiz+ROedYyRKSnOJOQLff82zqXMuNE\n5H0R+V5ENorITV6bI0UkznO+Dz3DhSAij4vIUhFZJSJvnFxvjD9ZgjGmeFoD7wHn4wwC+hdV7QbE\nA//wDAT6Os6glmfjTDyVn3VAX1XtijOkzL/yKNcJuBA4C3hcRBp61ncF7saZCK45znAhAJNUtYeq\ndsAZefiiQl+pMYVkY5EZUzy/q+oSEbkI50P9J0/lIAxYjDN67mZV/c1T/mNgTD7Hqw68KyIxOOO8\nheZRbrqqHgOOich8nIniDgC/qmoygGf8qWhgEdBfRO4HIoAzgNXAzKJdsjG+sQRjTPEc8fwrwDeq\nOsJ7o4h0LeTxngDmq+olnkmiFuRRLucggieXT3itywJCPLWoV4FYVd0mIuNwRsA2xq+sicyYkrEE\n6CMiLQFEJEJEWuE0eTX3JAuAKwo4TnVgu+f1qHzKDRWRcBGphTOib37zBZ1MJns9/ULDC4jBmBJh\nCcaYEqCqKTgJ4WMRScRJOG08zVi3AnNFZBGwGziYz6EmAk+LyE840yXk5VfgK895nlDVPCeJ8syG\nOhln2t5plN7kdaaCs+H6jfEzEYlU1TTPnVuvABtV9YViHG8ckKaqz5VUjMb4g9VgjPG/mzwd7qtx\nmsBedzkeY0qF1WCMcYGIjAbuyrH6J1W9zY14jPEHSzDGGGP8wprIjDHG+IUlGGOMMX5hCcYYY4xf\nWIIxxhjjF5ZgjDHG+MX/A4BSP0+W4kLFAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x6424e80>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# summarize results\n",
    "print(\"Best: %f using %s\" % (gsearch5_1.best_score_, gsearch5_1.best_params_))\n",
    "test_means = gsearch5_1.cv_results_[ 'mean_test_score' ]\n",
    "test_stds = gsearch5_1.cv_results_[ 'std_test_score' ]\n",
    "train_means = gsearch5_1.cv_results_[ 'mean_train_score' ]\n",
    "train_stds = gsearch5_1.cv_results_[ 'std_train_score' ]\n",
    "\n",
    "pd.DataFrame(gsearch5_1.cv_results_).to_csv('my_preds_reg_alpha_reg_lambda_1.csv')\n",
    "\n",
    "# plot results\n",
    "test_scores = np.array(test_means).reshape(len(reg_alpha), len(reg_lambda))\n",
    "train_scores = np.array(train_means).reshape(len(reg_alpha), len(reg_lambda))\n",
    "\n",
    "#log_reg_alpha = [0,0,0,0]\n",
    "#for index in range(len(reg_alpha)):\n",
    "#   log_reg_alpha[index] = math.log10(reg_alpha[index])\n",
    "    \n",
    "for i, value in enumerate(reg_alpha):\n",
    "    pyplot.plot(reg_lambda, -test_scores[i], label= 'reg_alpha:'   + str(value))\n",
    "#for i, value in enumerate(min_child_weight):\n",
    "#    pyplot.plot(max_depth, train_scores[i], label= 'train_min_child_weight:'   + str(value))\n",
    "    \n",
    "pyplot.legend()\n",
    "pyplot.xlabel( 'reg_alpha' )                                                                                                      \n",
    "pyplot.ylabel( '-Log Loss' )\n",
    "pyplot.savefig( 'reg_alpha_vs_reg_lambda1.png' )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "reg_alpha是1.5， 在边界点，再修改测试"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'reg_alpha': [0, 0.1, 1, 1.5]}"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "reg_alpha = [0, 0.1, 1, 1.5]    #default = 0, 测试0.1,1，1.5，2\n",
    "\n",
    "param_test5_1 = dict(reg_alpha=reg_alpha)\n",
    "param_test5_1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Anaconda\\lib\\site-packages\\sklearn\\model_selection\\_search.py:761: DeprecationWarning: The grid_scores_ attribute was deprecated in version 0.18 in favor of the more elaborate cv_results_ attribute. The grid_scores_ attribute will not be available from 0.20\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "([mean: -0.58152, std: 0.00365, params: {'reg_alpha': 0},\n",
       "  mean: -0.58156, std: 0.00363, params: {'reg_alpha': 0.1},\n",
       "  mean: -0.58119, std: 0.00321, params: {'reg_alpha': 1},\n",
       "  mean: -0.58127, std: 0.00342, params: {'reg_alpha': 1.5}],\n",
       " {'reg_alpha': 1},\n",
       " -0.58119388180544984)"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xgb5_1 = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=218,  #第二轮参数调整得到的n_estimators最优值\n",
    "        max_depth=6,\n",
    "        min_child_weight=4,\n",
    "        gamma=0,\n",
    "        subsample=0.8,\n",
    "        colsample_bytree=0.6,\n",
    "        colsample_bylevel = 0.7,\n",
    "        reg_lambda = 1,\n",
    "        objective= 'multi:softprob',\n",
    "        seed=3)\n",
    "\n",
    "\n",
    "gsearch5_1 = GridSearchCV(xgb5_1, param_grid = param_test5_1, scoring='neg_log_loss',n_jobs=-1, cv=kfold)\n",
    "gsearch5_1.fit(X_train , y_train)\n",
    "\n",
    "gsearch5_1.grid_scores_, gsearch5_1.best_params_,     gsearch5_1.best_score_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 保存模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<bound method XGBClassifier.get_xgb_params of XGBClassifier(base_score=0.5, booster='gbtree', colsample_bylevel=0.7,\n",
       "       colsample_bytree=0.6, gamma=0, learning_rate=0.1, max_delta_step=0,\n",
       "       max_depth=6, min_child_weight=4, missing=None, n_estimators=218,\n",
       "       n_jobs=1, nthread=None, objective='multi:softprob', random_state=0,\n",
       "       reg_alpha=0, reg_lambda=1, scale_pos_weight=1, seed=3, silent=True,\n",
       "       subsample=0.8)>"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import cPickle\n",
    "cPickle.dump(xgb5_1, open(\"xgb_model.pkl\", 'wb'))\n",
    "xgb5_1.get_xgb_params"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
